{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "7fb12917",
   "metadata": {},
   "source": [
    "| [04_text_classification/01_机器学习分类模型.ipynb](https://github.com/shibing624/nlp-tutorial/tree/main/04_text_classification/01_机器学习分类模型.ipynb)  | 基于scikit-learn训练LR等传统机器学习模型  |[Open In Colab](https://colab.research.google.com/github/shibing624/nlp-tutorial/blob/main/04_text_classification/01_机器学习分类模型.ipynb) |\n",
    "\n",
    "# 文本分类\n",
    "\n",
    "经典的文本分类模型是使用scikit-learn提取文本bag-of-words向量，然后提取tfidf特征，后接逻辑回归（LR）模型。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "08b31292",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Automatically created module for IPython interactive environment\n",
      "Usage: ipykernel_launcher.py [options]\n",
      "\n",
      "Options:\n",
      "  -h, --help            show this help message and exit\n",
      "  --report              Print a detailed classification report.\n",
      "  --chi2_select=SELECT_CHI2\n",
      "                        Select some number of features using a chi-squared\n",
      "                        test\n",
      "  --confusion_matrix    Print the confusion matrix.\n",
      "  --top10               Print ten most discriminative terms per class for\n",
      "                        every classifier.\n",
      "  --all_categories      Whether to use all categories or not.\n",
      "  --use_hashing         Use a hashing vectorizer.\n",
      "  --n_features=N_FEATURES\n",
      "                        n_features when using the hashing vectorizer.\n",
      "  --filtered            Remove newsgroup information that is easily overfit:\n",
      "                        headers, signatures, and quoting.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import logging\n",
    "import numpy as np\n",
    "from optparse import OptionParser\n",
    "import sys\n",
    "from time import time\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from sklearn.datasets import fetch_20newsgroups\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.feature_extraction.text import HashingVectorizer\n",
    "from sklearn.feature_selection import SelectFromModel\n",
    "from sklearn.feature_selection import SelectKBest, chi2\n",
    "from sklearn.linear_model import RidgeClassifier\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.linear_model import PassiveAggressiveClassifier\n",
    "from sklearn.naive_bayes import BernoulliNB, MultinomialNB\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.neighbors import NearestCentroid\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.utils.extmath import density\n",
    "from sklearn import metrics\n",
    "\n",
    "# Display progress logs on stdout\n",
    "logging.basicConfig(level=logging.INFO,\n",
    "                    format='%(asctime)s %(levelname)s %(message)s')\n",
    "\n",
    "# parse commandline arguments\n",
    "op = OptionParser()\n",
    "op.add_option(\"--report\", default=True,\n",
    "              action=\"store_true\", dest=\"print_report\",\n",
    "              help=\"Print a detailed classification report.\")\n",
    "op.add_option(\"--chi2_select\",\n",
    "              action=\"store\", type=\"int\", dest=\"select_chi2\",\n",
    "              help=\"Select some number of features using a chi-squared test\")\n",
    "op.add_option(\"--confusion_matrix\",\n",
    "              action=\"store_true\", dest=\"print_cm\",\n",
    "              help=\"Print the confusion matrix.\")\n",
    "op.add_option(\"--top10\",\n",
    "              action=\"store_true\", dest=\"print_top10\",\n",
    "              help=\"Print ten most discriminative terms per class\"\n",
    "                   \" for every classifier.\")\n",
    "op.add_option(\"--all_categories\", default=False,\n",
    "              action=\"store_true\", dest=\"all_categories\",\n",
    "              help=\"Whether to use all categories or not.\")\n",
    "op.add_option(\"--use_hashing\",\n",
    "              action=\"store_true\", default=False,\n",
    "              help=\"Use a hashing vectorizer.\")\n",
    "op.add_option(\"--n_features\",\n",
    "              action=\"store\", type=int, default=2 ** 16,\n",
    "              help=\"n_features when using the hashing vectorizer.\")\n",
    "op.add_option(\"--filtered\",\n",
    "              action=\"store_true\", default=True,\n",
    "              help=\"Remove newsgroup information that is easily overfit: \"\n",
    "                   \"headers, signatures, and quoting.\")\n",
    "\n",
    "\n",
    "def is_interactive():\n",
    "    return not hasattr(sys.modules['__main__'], '__file__')\n",
    "\n",
    "\n",
    "# work-around for Jupyter notebook and IPython console\n",
    "argv = [] if is_interactive() else sys.argv[1:]\n",
    "(opts, args) = op.parse_args(argv)\n",
    "if len(args) < 0:\n",
    "    op.error(\"this script takes no arguments.\")\n",
    "    sys.exit(1)\n",
    "\n",
    "print(__doc__)\n",
    "op.print_help()\n",
    "print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "5c96e4d5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading 20 newsgroups dataset for categories:\n",
      "['alt.atheism', 'talk.religion.misc', 'comp.graphics', 'sci.space']\n",
      "data loaded\n"
     ]
    }
   ],
   "source": [
    "# Load some categories from the training set\n",
    "if opts.all_categories:\n",
    "    categories = None\n",
    "else:\n",
    "    categories = [\n",
    "        'alt.atheism',\n",
    "        'talk.religion.misc',\n",
    "        'comp.graphics',\n",
    "        'sci.space',\n",
    "    ]\n",
    "\n",
    "if opts.filtered:\n",
    "    remove = ('headers', 'footers', 'quotes')\n",
    "else:\n",
    "    remove = ()\n",
    "\n",
    "print(\"Loading 20 newsgroups dataset for categories:\")\n",
    "print(categories if categories else \"all\")\n",
    "\n",
    "data_train = fetch_20newsgroups(subset='train', categories=categories,\n",
    "                                shuffle=True, random_state=42,\n",
    "                                remove=remove)\n",
    "\n",
    "data_test = fetch_20newsgroups(subset='test', categories=categories,\n",
    "                               shuffle=True, random_state=42,\n",
    "                               remove=remove)\n",
    "print('data loaded')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "9e10163a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2034 documents - 2.428MB (training set)\n",
      "1353 documents - 1.800MB (test set)\n",
      "4 categories\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# order of labels in `target_names` can be different from `categories`\n",
    "target_names = data_train.target_names\n",
    "\n",
    "\n",
    "def size_mb(docs):\n",
    "    return sum(len(s.encode('utf-8')) for s in docs) / 1e6\n",
    "\n",
    "\n",
    "data_train_size_mb = size_mb(data_train.data)\n",
    "data_test_size_mb = size_mb(data_test.data)\n",
    "\n",
    "print(\"%d documents - %0.3fMB (training set)\" % (\n",
    "    len(data_train.data), data_train_size_mb))\n",
    "print(\"%d documents - %0.3fMB (test set)\" % (\n",
    "    len(data_test.data), data_test_size_mb))\n",
    "print(\"%d categories\" % len(categories))\n",
    "print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "313366cd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting features from the training data using a sparse vectorizer\n",
      "done in 0.248536s at 9.769MB/s\n",
      "n_samples: 2034, n_features: 26576\n",
      "\n",
      "Extracting features from the test data using the same vectorizer\n",
      "done in 0.155689s at 11.559MB/s\n",
      "n_samples: 1353, n_features: 26576\n",
      "\n",
      "feature_names.shape: (26576,)\n",
      "feature_names: ['00' '000' '0000' ... 'zware' 'zwarte' 'zyxel']\n"
     ]
    }
   ],
   "source": [
    "# split a training set and a test set\n",
    "y_train, y_test = data_train.target, data_test.target\n",
    "\n",
    "print(\"Extracting features from the training data using a sparse vectorizer\")\n",
    "t0 = time()\n",
    "if opts.use_hashing:\n",
    "    vectorizer = HashingVectorizer(stop_words='english', alternate_sign=False,\n",
    "                                   n_features=opts.n_features)\n",
    "    X_train = vectorizer.transform(data_train.data)\n",
    "else:\n",
    "    vectorizer = TfidfVectorizer(sublinear_tf=True, max_df=0.5,\n",
    "                                 stop_words='english')\n",
    "    X_train = vectorizer.fit_transform(data_train.data)\n",
    "duration = time() - t0\n",
    "print(\"done in %fs at %0.3fMB/s\" % (duration, data_train_size_mb / duration))\n",
    "print(\"n_samples: %d, n_features: %d\" % X_train.shape)\n",
    "print()\n",
    "\n",
    "print(\"Extracting features from the test data using the same vectorizer\")\n",
    "t0 = time()\n",
    "X_test = vectorizer.transform(data_test.data)\n",
    "duration = time() - t0\n",
    "print(\"done in %fs at %0.3fMB/s\" % (duration, data_test_size_mb / duration))\n",
    "print(\"n_samples: %d, n_features: %d\" % X_test.shape)\n",
    "print()\n",
    "\n",
    "# mapping from integer feature name to original token string\n",
    "if opts.use_hashing:\n",
    "    feature_names = None\n",
    "else:\n",
    "    feature_names = vectorizer.get_feature_names()\n",
    "\n",
    "if opts.select_chi2:\n",
    "    print(\"Extracting %d best features by a chi-squared test\" %\n",
    "          opts.select_chi2)\n",
    "    t0 = time()\n",
    "    ch2 = SelectKBest(chi2, k=opts.select_chi2)\n",
    "    X_train = ch2.fit_transform(X_train, y_train)\n",
    "    X_test = ch2.transform(X_test)\n",
    "    print(\"X-train shape:\", X_train.shape)\n",
    "    print(\"X-test shape:\", X_test.shape)\n",
    "    if feature_names:\n",
    "        # keep selected feature names\n",
    "        feature_names = [feature_names[i] for i\n",
    "                         in ch2.get_support(indices=True)]\n",
    "    print(\"done in %fs\" % (time() - t0))\n",
    "    print()\n",
    "\n",
    "if feature_names:\n",
    "    feature_names = np.asarray(feature_names)\n",
    "    print('feature_names.shape:', feature_names.shape)\n",
    "    print('feature_names:', feature_names)\n",
    "\n",
    "\n",
    "def trim(s):\n",
    "    \"\"\"Trim string to fit on terminal (assuming 80-column display)\"\"\"\n",
    "    return s if len(s) <= 80 else s[:77] + \"...\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "542ac768",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================================================\n",
      "LogisticRegression\n",
      "________________________________________________________________________________\n",
      "Training: \n",
      "LogisticRegression()\n",
      "train time: 0.823s\n",
      "test time:  0.001s\n",
      "accuracy:   0.753\n",
      "dimensionality: 26576\n",
      "density: 1.000000\n",
      "\n",
      "classification report:\n",
      "                    precision    recall  f1-score   support\n",
      "\n",
      "       alt.atheism       0.65      0.61      0.63       319\n",
      "     comp.graphics       0.89      0.90      0.89       389\n",
      "         sci.space       0.72      0.91      0.81       394\n",
      "talk.religion.misc       0.70      0.47      0.56       251\n",
      "\n",
      "          accuracy                           0.75      1353\n",
      "         macro avg       0.74      0.72      0.72      1353\n",
      "      weighted avg       0.75      0.75      0.74      1353\n",
      "\n",
      "\n",
      "================================================================================\n",
      "kNN\n",
      "________________________________________________________________________________\n",
      "Training: \n",
      "KNeighborsClassifier(n_neighbors=10)\n",
      "train time: 0.001s\n",
      "test time:  0.135s\n",
      "accuracy:   0.256\n",
      "classification report:\n",
      "                    precision    recall  f1-score   support\n",
      "\n",
      "       alt.atheism       0.24      0.31      0.27       319\n",
      "     comp.graphics       0.29      0.27      0.28       389\n",
      "         sci.space       0.33      0.21      0.25       394\n",
      "talk.religion.misc       0.19      0.24      0.21       251\n",
      "\n",
      "          accuracy                           0.26      1353\n",
      "         macro avg       0.26      0.26      0.25      1353\n",
      "      weighted avg       0.27      0.26      0.26      1353\n",
      "\n",
      "\n",
      "================================================================================\n",
      "Random forest\n",
      "________________________________________________________________________________\n",
      "Training: \n",
      "RandomForestClassifier()\n",
      "train time: 1.555s\n",
      "test time:  0.067s\n",
      "accuracy:   0.715\n",
      "classification report:\n",
      "                    precision    recall  f1-score   support\n",
      "\n",
      "       alt.atheism       0.59      0.63      0.61       319\n",
      "     comp.graphics       0.77      0.91      0.83       389\n",
      "         sci.space       0.79      0.79      0.79       394\n",
      "talk.religion.misc       0.65      0.41      0.51       251\n",
      "\n",
      "          accuracy                           0.72      1353\n",
      "         macro avg       0.70      0.68      0.68      1353\n",
      "      weighted avg       0.71      0.72      0.71      1353\n",
      "\n",
      "\n",
      "================================================================================\n",
      "L2 penalty\n",
      "________________________________________________________________________________\n",
      "Training: \n",
      "LinearSVC(dual=False, tol=0.001)\n",
      "train time: 0.047s\n",
      "test time:  0.001s\n",
      "accuracy:   0.780\n",
      "dimensionality: 26576\n",
      "density: 1.000000\n",
      "\n",
      "classification report:\n",
      "                    precision    recall  f1-score   support\n",
      "\n",
      "       alt.atheism       0.69      0.62      0.66       319\n",
      "     comp.graphics       0.89      0.91      0.90       389\n",
      "         sci.space       0.78      0.90      0.84       394\n",
      "talk.religion.misc       0.68      0.60      0.64       251\n",
      "\n",
      "          accuracy                           0.78      1353\n",
      "         macro avg       0.76      0.76      0.76      1353\n",
      "      weighted avg       0.77      0.78      0.78      1353\n",
      "\n",
      "\n",
      "________________________________________________________________________________\n",
      "Training: \n",
      "SGDClassifier()\n",
      "train time: 0.021s\n",
      "test time:  0.001s\n",
      "accuracy:   0.774\n",
      "dimensionality: 26576\n",
      "density: 0.601520\n",
      "\n",
      "classification report:\n",
      "                    precision    recall  f1-score   support\n",
      "\n",
      "       alt.atheism       0.67      0.63      0.65       319\n",
      "     comp.graphics       0.89      0.89      0.89       389\n",
      "         sci.space       0.78      0.89      0.84       394\n",
      "talk.religion.misc       0.67      0.58      0.62       251\n",
      "\n",
      "          accuracy                           0.77      1353\n",
      "         macro avg       0.76      0.75      0.75      1353\n",
      "      weighted avg       0.77      0.77      0.77      1353\n",
      "\n",
      "\n",
      "================================================================================\n",
      "L1 penalty\n",
      "________________________________________________________________________________\n",
      "Training: \n",
      "LinearSVC(dual=False, penalty='l1', tol=0.001)\n",
      "train time: 0.095s\n",
      "test time:  0.001s\n",
      "accuracy:   0.746\n",
      "dimensionality: 26576\n",
      "density: 0.013113\n",
      "\n",
      "classification report:\n",
      "                    precision    recall  f1-score   support\n",
      "\n",
      "       alt.atheism       0.68      0.59      0.63       319\n",
      "     comp.graphics       0.88      0.87      0.87       389\n",
      "         sci.space       0.73      0.88      0.80       394\n",
      "talk.religion.misc       0.64      0.55      0.59       251\n",
      "\n",
      "          accuracy                           0.75      1353\n",
      "         macro avg       0.73      0.72      0.72      1353\n",
      "      weighted avg       0.74      0.75      0.74      1353\n",
      "\n",
      "\n",
      "________________________________________________________________________________\n",
      "Training: \n",
      "SGDClassifier(penalty='l1')\n",
      "train time: 0.070s\n",
      "test time:  0.001s\n",
      "accuracy:   0.744\n",
      "dimensionality: 26576\n",
      "density: 0.041767\n",
      "\n",
      "classification report:\n",
      "                    precision    recall  f1-score   support\n",
      "\n",
      "       alt.atheism       0.66      0.58      0.62       319\n",
      "     comp.graphics       0.88      0.87      0.88       389\n",
      "         sci.space       0.76      0.85      0.80       394\n",
      "talk.religion.misc       0.61      0.59      0.60       251\n",
      "\n",
      "          accuracy                           0.74      1353\n",
      "         macro avg       0.73      0.72      0.72      1353\n",
      "      weighted avg       0.74      0.74      0.74      1353\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Benchmark classifiers\n",
    "def benchmark(clf):\n",
    "    print('_' * 80)\n",
    "    print(\"Training: \")\n",
    "    print(clf)\n",
    "    t0 = time()\n",
    "    clf.fit(X_train, y_train)\n",
    "    train_time = time() - t0\n",
    "    print(\"train time: %0.3fs\" % train_time)\n",
    "\n",
    "    t0 = time()\n",
    "    pred = clf.predict(X_test)\n",
    "    test_time = time() - t0\n",
    "    print(\"test time:  %0.3fs\" % test_time)\n",
    "\n",
    "    score = metrics.accuracy_score(y_test, pred)\n",
    "    print(\"accuracy:   %0.3f\" % score)\n",
    "\n",
    "    if hasattr(clf, 'coef_'):\n",
    "        print(\"dimensionality: %d\" % clf.coef_.shape[1])\n",
    "        print(\"density: %f\" % density(clf.coef_))\n",
    "\n",
    "        if opts.print_top10 and feature_names is not None:\n",
    "            print(\"top 10 keywords per class:\")\n",
    "            for i, label in enumerate(target_names):\n",
    "                top10 = np.argsort(clf.coef_[i])[-10:]\n",
    "                print(trim(\"%s: %s\" % (label, \" \".join(feature_names[top10]))))\n",
    "        print()\n",
    "\n",
    "    if opts.print_report:\n",
    "        print(\"classification report:\")\n",
    "        print(metrics.classification_report(y_test, pred,\n",
    "                                            target_names=target_names))\n",
    "\n",
    "    if opts.print_cm:\n",
    "        print(\"confusion matrix:\")\n",
    "        print(metrics.confusion_matrix(y_test, pred))\n",
    "\n",
    "    print()\n",
    "    clf_descr = str(clf).split('(')[0]\n",
    "    return clf_descr, score, train_time, test_time\n",
    "\n",
    "\n",
    "results = []\n",
    "for clf, name in (\n",
    "    (LogisticRegression(), \"LogisticRegression\"),\n",
    "    (KNeighborsClassifier(n_neighbors=10), \"kNN\"),\n",
    "    (RandomForestClassifier(n_estimators=100), \"Random forest\")):\n",
    "    print('=' * 80)\n",
    "    print(name)\n",
    "    results.append(benchmark(clf))\n",
    "\n",
    "for penalty in [\"l2\", \"l1\"]:\n",
    "    print('=' * 80)\n",
    "    print(\"%s penalty\" % penalty.upper())\n",
    "    # Train Liblinear model\n",
    "    results.append(benchmark(LinearSVC(penalty=penalty, dual=False,\n",
    "                                       tol=1e-3)))\n",
    "\n",
    "    # Train SGD model\n",
    "    results.append(benchmark(SGDClassifier(alpha=.0001,\n",
    "                                           penalty=penalty)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "b382b60c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================================================\n",
      "Elastic-Net penalty\n",
      "________________________________________________________________________________\n",
      "Training: \n",
      "SGDClassifier(penalty='elasticnet')\n",
      "train time: 0.112s\n",
      "test time:  0.001s\n",
      "accuracy:   0.767\n",
      "dimensionality: 26576\n",
      "density: 0.230556\n",
      "\n",
      "classification report:\n",
      "                    precision    recall  f1-score   support\n",
      "\n",
      "       alt.atheism       0.70      0.61      0.65       319\n",
      "     comp.graphics       0.88      0.88      0.88       389\n",
      "         sci.space       0.78      0.88      0.82       394\n",
      "talk.religion.misc       0.65      0.62      0.63       251\n",
      "\n",
      "          accuracy                           0.77      1353\n",
      "         macro avg       0.75      0.75      0.75      1353\n",
      "      weighted avg       0.76      0.77      0.76      1353\n",
      "\n",
      "\n",
      "================================================================================\n",
      "NearestCentroid (aka Rocchio classifier)\n",
      "________________________________________________________________________________\n",
      "Training: \n",
      "NearestCentroid()\n",
      "train time: 0.006s\n",
      "test time:  0.003s\n",
      "accuracy:   0.756\n",
      "classification report:\n",
      "                    precision    recall  f1-score   support\n",
      "\n",
      "       alt.atheism       0.67      0.57      0.61       319\n",
      "     comp.graphics       0.91      0.86      0.88       389\n",
      "         sci.space       0.75      0.90      0.82       394\n",
      "talk.religion.misc       0.63      0.61      0.62       251\n",
      "\n",
      "          accuracy                           0.76      1353\n",
      "         macro avg       0.74      0.73      0.73      1353\n",
      "      weighted avg       0.75      0.76      0.75      1353\n",
      "\n",
      "\n",
      "================================================================================\n",
      "Naive Bayes\n",
      "________________________________________________________________________________\n",
      "Training: \n",
      "MultinomialNB(alpha=0.01)\n",
      "train time: 0.005s\n",
      "test time:  0.001s\n",
      "accuracy:   0.788\n",
      "dimensionality: 26576\n",
      "density: 1.000000\n",
      "\n",
      "classification report:\n",
      "                    precision    recall  f1-score   support\n",
      "\n",
      "       alt.atheism       0.69      0.67      0.68       319\n",
      "     comp.graphics       0.92      0.89      0.90       389\n",
      "         sci.space       0.80      0.90      0.85       394\n",
      "talk.religion.misc       0.69      0.60      0.64       251\n",
      "\n",
      "          accuracy                           0.79      1353\n",
      "         macro avg       0.77      0.77      0.77      1353\n",
      "      weighted avg       0.79      0.79      0.79      1353\n",
      "\n",
      "\n",
      "________________________________________________________________________________\n",
      "Training: \n",
      "BernoulliNB(alpha=0.01)\n",
      "train time: 0.004s\n",
      "test time:  0.003s\n",
      "accuracy:   0.722\n",
      "dimensionality: 26576\n",
      "density: 1.000000\n",
      "\n",
      "classification report:\n",
      "                    precision    recall  f1-score   support\n",
      "\n",
      "       alt.atheism       0.60      0.75      0.67       319\n",
      "     comp.graphics       0.72      0.95      0.82       389\n",
      "         sci.space       0.90      0.69      0.78       394\n",
      "talk.religion.misc       0.70      0.39      0.51       251\n",
      "\n",
      "          accuracy                           0.72      1353\n",
      "         macro avg       0.73      0.69      0.69      1353\n",
      "      weighted avg       0.74      0.72      0.71      1353\n",
      "\n",
      "\n",
      "================================================================================\n",
      "LinearSVC with L1-based feature selection\n",
      "________________________________________________________________________________\n",
      "Training: \n",
      "Pipeline(steps=[('feature_selection',\n",
      "                 SelectFromModel(estimator=LinearSVC(dual=False, penalty='l1',\n",
      "                                                     tol=0.001))),\n",
      "                ('classification', LinearSVC())])\n",
      "train time: 0.135s\n",
      "test time:  0.003s\n",
      "accuracy:   0.745\n",
      "classification report:\n",
      "                    precision    recall  f1-score   support\n",
      "\n",
      "       alt.atheism       0.67      0.58      0.62       319\n",
      "     comp.graphics       0.86      0.87      0.87       389\n",
      "         sci.space       0.74      0.87      0.80       394\n",
      "talk.religion.misc       0.64      0.56      0.60       251\n",
      "\n",
      "          accuracy                           0.75      1353\n",
      "         macro avg       0.73      0.72      0.72      1353\n",
      "      weighted avg       0.74      0.75      0.74      1353\n",
      "\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/xuming/opt/anaconda3/lib/python3.8/site-packages/sklearn/utils/deprecation.py:101: FutureWarning: Attribute coef_ was deprecated in version 0.24 and will be removed in 1.1 (renaming of 0.26).\n",
      "  warnings.warn(msg, category=FutureWarning)\n",
      "/Users/xuming/opt/anaconda3/lib/python3.8/site-packages/sklearn/utils/deprecation.py:101: FutureWarning: Attribute coef_ was deprecated in version 0.24 and will be removed in 1.1 (renaming of 0.26).\n",
      "  warnings.warn(msg, category=FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "# Train SGD with Elastic Net penalty\n",
    "print('=' * 80)\n",
    "print(\"Elastic-Net penalty\")\n",
    "results.append(benchmark(SGDClassifier(alpha=.0001, penalty=\"elasticnet\")))\n",
    "\n",
    "# Train NearestCentroid without threshold\n",
    "print('=' * 80)\n",
    "print(\"NearestCentroid (aka Rocchio classifier)\")\n",
    "results.append(benchmark(NearestCentroid()))\n",
    "\n",
    "# Train sparse Naive Bayes classifiers\n",
    "print('=' * 80)\n",
    "print(\"Naive Bayes\")\n",
    "results.append(benchmark(MultinomialNB(alpha=.01)))\n",
    "results.append(benchmark(BernoulliNB(alpha=.01)))\n",
    "\n",
    "print('=' * 80)\n",
    "print(\"LinearSVC with L1-based feature selection\")\n",
    "# The smaller C, the stronger the regularization.\n",
    "# The more regularization, the more sparsity.\n",
    "results.append(benchmark(Pipeline([\n",
    "    ('feature_selection', SelectFromModel(LinearSVC(penalty=\"l1\", dual=False,\n",
    "                                                    tol=1e-3))),\n",
    "    ('classification', LinearSVC(penalty=\"l2\"))])))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "b74ddffe",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAuAAAAI1CAYAAACXLU+VAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAABHIUlEQVR4nO3de7zmY73/8dcbE4aJQrbpYCSnMgzL6CCMkkLH3Vkn2kKUtLE77J1D7UobHbBldxC1aUtUKjuTtklErMXkHCpJ9oP4hZlpxjbj8/vj/o5uY82se40137Vm5vV8PNZj3ff1vb7X9/O9b+PxXte67mulqpAkSZLUjlVGuwBJkiRpZWIAlyRJklpkAJckSZJaZACXJEmSWmQAlyRJklpkAJckSZJaZACXJEmSWmQAlyQt15K8NMkvkzyY5P8luTzJ1NGuS5IWZ7XRLkCSpKWV5KnAj4D3A98BngLsDDw8gtdYtaoWjNR4kuQMuCRpebY5QFV9u6oWVNXcqppeVdcBJHlfkpuTzEpyU5Ltm/atksxI8kCSG5O8duGASc5I8uUkFyaZA+yWZGKS85L8Ocnvkxw6KncraYVgAJckLc9uBRYkOTPJnkmetvBAkjcDxwDvBp4KvBa4P8k44IfAdOAZwAeBs5Js0TXuPsCngQnAL5v+vwaeCbwcOCzJK5fxvUlaQRnAJUnLrap6CHgpUMBXgT8nuSDJhsD+wL9V1dXVcXtV/QF4EbA2cFxV/V9V/Q+dZSxv7xr6B1V1eVU9CkwGNqiqTzb9f9dc623t3amkFYlrwCVJy7WquhnYFyDJlsB/Al8Eng38dpBTJgJ/bML1Qn+gM7u90B+7Hm8MTEzyQFfbqsAvnmTpklZSBnBJ0gqjqm5JcgZwIJ0Qvekg3e4Gnp1kla4Q/hw6y1keG6rr8R+B31fVZsugZEkrIZegSJKWW0m2THJ4kmc1z59NZynJlcDXgCOS9KXjeUk2Bn4FzAH+Kcm4JNOA1wD/tZjLXAU8lOQjSdZMsmqSrd3qUNLSMoBLkpZns4AXAr9qdiy5ErgBOLyqzqXzQcqzm37fB55eVf9H5wOZewL3AacC766qWwa7QLMF4WuAKcDvm3O+BqyzzO5K0gotVTV0L0mSJEkjwhlwSZIkqUUGcEmSJKlFBnBJkiSpRQZwSZIkqUXuA64xbf31169JkyaNdhmSJEnDMjAwcF9VbTDYMQO4xrRJkybR398/2mVIkiQNS5I/LO6YS1AkSZKkFhnAJUmSpBYZwCVJkqQWuQZckiRpOfPII49w1113MW/evNEuZaW3xhpr8KxnPYtx48b1fI4BXJIkaTlz1113MWHCBCZNmkSS0S5npVVV3H///dx1111ssskmPZ/nEhRJkqTlzLx581hvvfUM36MsCeutt96wfxNhAJckSVoOGb7HhqV5HwzgkiRJUotcAy5JkrScS44d0fGqjh7R8fR4zoBLkiRp1MyfP3+0S2idAVySJEnDMmfOHPbee2+23XZbtt56a8455xyuvvpqXvKSl7Dtttuy4447MmvWLObNm8d+++3H5MmT2W677bjkkksAOOOMM3jzm9/Ma17zGvbYYw/mzJnDe9/7XqZOncp2223HD37wg1G+w2XLJSiSJEkalp/85CdMnDiRH//4xwA8+OCDbLfddpxzzjlMnTqVhx56iDXXXJMvfelLAFx//fXccsst7LHHHtx6660AXHHFFVx33XU8/elP5+Mf/zgve9nLOP3003nggQfYcccd2X333VlrrbVG7R6XJWfAJUmSNCyTJ0/m4osv5iMf+Qi/+MUvuPPOO9loo42YOnUqAE996lNZbbXVuOyyy3jXu94FwJZbbsnGG2/8WAB/xStewdOf/nQApk+fznHHHceUKVOYNm0a8+bN48477xydm2uBM+CSJEkals0335yBgQEuvPBCPvaxj7HHHnsMuh1fVS12jO7Z7arivPPOY4sttlgm9Y41zoBLkiRpWO6++27Gjx/PO9/5To444giuvPJK7r77bq6++moAZs2axfz589lll10466yzALj11lu58847Bw3Zr3zlKzn55JMfC+zXXnttezczCpwBlyRJWs61vW3g9ddfz5FHHskqq6zCuHHj+PKXv0xV8cEPfpC5c+ey5pprcvHFF3PwwQdz0EEHMXnyZFZbbTXOOOMMVl999SeM94lPfILDDjuMbbbZhqpi0qRJ/OhHP2r1ntqUJf1qQBptO+ywQ/X39492GZIkjSk333wzW2211WiXocZg70eSgaraYbD+LkGRJEmSWmQAlyRJklpkAJckSZJaZACXJEmSWmQAlyRJklrkNoQa2+4ZgBOfuLH/Exzubj6SJGn5YACXJElazmXGjBEdr6ZNW+LxBx54gLPPPpuDDz542GPvtddenH322ay77rqL7XPUUUexyy67sPvuuw97/EV95jOf4eMf//hjz1/ykpfwy1/+8kmP+2S4BEWSJEnD8sADD3DqqacOemzBggVLPPfCCy9cYvgG+OQnPzki4Rs6AbzbaIdvMIBLkiRpmD760Y/y29/+lilTpnDkkUcyY8YMdtttN/bZZx8mT54MwOtf/3r6+vp4wQtewFe+8pXHzp00aRL33Xcfd9xxB1tttRXve9/7eMELXsAee+zB3LlzAdh333357ne/+1j/o48+mu23357Jkydzyy23APDnP/+ZV7ziFWy//fYceOCBbLzxxtx3331PqHPu3LlMmTKFd7zjHQCsvfbaAMyYMYNdd92Vt7zlLWy++eZ89KMf5ayzzmLHHXdk8uTJ/Pa3v33sOm984xuZOnUqU6dO5fLLL3/Sr58BXJIkScNy3HHHsemmmzJz5kyOP/54AK666io+/elPc9NNNwFw+umnMzAwQH9/PyeddBL333//E8a57bbbOOSQQ7jxxhtZd911Oe+88wa93vrrr88111zD+9//fk444QQAjj32WF72spdxzTXX8IY3vIE777xz0DrXXHNNZs6cyVlnnfWE47/+9a/50pe+xPXXX8+3vvUtbr31Vq666ir2339/Tj75ZAA+9KEP8eEPf5irr76a8847j/3333/pXrQurgGXJEnSk7bjjjuyySabPPb8pJNO4nvf+x4Af/zjH7nttttYb731HnfOJptswpQpUwDo6+vjjjvuGHTsv//7v3+sz/nnnw/AZZdd9tj4r3rVq3ja05427JqnTp3KRhttBMCmm27KHnvsAcDkyZO55JJLALj44osf+6EC4KGHHmLWrFlMmDBh2NdbyAAuSZKkJ22ttdZ67PGMGTO4+OKLueKKKxg/fjzTpk1j3rx5Tzhn9dVXf+zxqquu+tgSlMX1W3XVVZk/fz4AVU9+B7Tu66+yyiqPPV9llVUeu86jjz7KFVdcwZprrvmkr/fYtUZsJEmSJK0UJkyYwKxZsxZ7/MEHH+RpT3sa48eP55ZbbuHKK68c8Rpe+tKX8p3vfAeA6dOn85e//GXQfuPGjeORRx5Z6uvssccenHLKKY89nzlz5lKPtZAz4BrbNuyDw/tHuwpJksa0obYNHGnrrbceO+20E1tvvTV77rkne++99+OOv+pVr+K0005jm222YYsttuBFL3rRiNdw9NFH8/a3v51zzjmHXXfdlY022mjQZSEHHHAA22yzDdtvv/2g68CHctJJJ3HIIYewzTbbMH/+fHbZZRdOO+20J1V7RmL6XlpWdthhh+rvN4BLktTt5ptvZqutthrtMkbVww8/zKqrrspqq63GFVdcwfvf//4RmZ1eGoO9H0kGqmqHwfo7Ay5JkqTlzp133slb3vIWHn30UZ7ylKfw1a9+dbRL6pkBXJIkScudzTbbjGuvvXa0y1gqBnCNaQOzZo34n9dtQ9tr8SRJ0vLDXVAkSZKkFhnAJUmSpBYZwCVJkqQWuQZckiRpeXdiRna8w5e8TfUDDzzA2WefzcEHH7xUw3/xi1/kgAMOYPz48UMe22uvvTj77LNZd911l+paY5Ez4JIkSRqWBx54gFNPPXWpz//iF7/IX//6156OXXjhhStU+AYDuCRJkobpox/9KL/97W+ZMmUKRx55JADHH388U6dOZZtttuHoo48GYM6cOey9995su+22bL311pxzzjmcdNJJ3H333ey2227stttujxt3sGOTJk3ivvvu44477mDLLbdk//33Z+utt+Yd73gHF198MTvttBObbbYZV1111WPXfO9738vUqVPZbrvt+MEPftDiK9Mbl6BIkiRpWI477jhuuOGGx/7y5PTp07ntttu46qqrqCpe+9rXcumll/LnP/+ZiRMn8uMf/xiABx98kHXWWYfPf/7zXHLJJay//vqPG/fQQw9d7DGA22+/nXPPPZevfOUrTJ06lbPPPpvLLruMCy64gM985jN8//vf59Of/jQve9nLOP3003nggQfYcccd2X333VlrrbWW+evSKwO4xrS+CRPod09tSZLGtOnTpzN9+nS22247AGbPns1tt93GzjvvzBFHHMFHPvIRXv3qV7Pzzjs/qetssskmTJ48GYAXvOAFvPzlLycJkydP5o477nislgsuuIATTjgBgHnz5nHnnXc+4U/Fj6YhA3iSBcD1Td+bgfcAzwfeXVWHLs1Fk8yuqrWTTAROqqo3Lc04kiRJGn1Vxcc+9jEOPPDAJxwbGBjgwgsv5GMf+xh77LEHRx111FJfZ/XVV3/s8SqrrPLY81VWWYX58+c/Vst5553HFltssdTXWdZ6WQM+t6qmVNXWwP8BB1VV/9KG725VdbfhW5IkafkyYcIEZs2a9djzV77ylZx++unMnj0bgD/96U/ce++93H333YwfP553vvOdHHHEEVxzzTWDnr+ksYfrla98JSeffDJVnZ1cxuKfqx/uEpRfANskmQYcUVWvTnIMsCnwTODZwL9V1VcBkhwJvAVYHfheVR3dPViSScCPqmrrJPsCrwXGN+N9r6r+qem3B3BsM85vgf2qavZwb1aSJGmFNMS2gSNtvfXWY6eddmLrrbdmzz335Pjjj+fmm2/mxS9+MQBrr702//mf/8ntt9/OkUceySqrrMK4ceP48pe/DMABBxzAnnvuyUYbbcQll1zyuLGXdKwXn/jEJzjssMPYZpttqComTZrEj370oyd/0yMoC386WGyHvy0XWQ04D/gJnaUo3QH8DcCLgLWAa4EXAlsDbwIOBAJcQCecX9o15iQeH8CPArYDHgZ+A7wUmAucD+xZVXOSfARYvao+OYKvg8aoZGJ1/hOSJGloi8z1rbBuvvnmMbWmeWU32PuRZKCqdhisfy8z4Gsmmdk8/gXwdeAli/T5QVXNBeYmuQTYkU543oNOIAdYG9gMuHQJ1/pZVT3YFH0TsDGwLp0155cnAXgKcEUPdUuSJEljTi8BfG5VTeluaIJwt0Wn0YvOrPdnq+o/hlHPw12PFzT1BfhpVb19GONIkiRJY9JI/SGe1yVZI8l6wDTgauAi4L1J1gZI8swkz1iKsa8EdkryvGac8Uk2H6G6JUmSlktDLSNWO5bmfRipfcCvAn4MPAf4VFXdDdydZCvgimbGfDbwTuDe4QxcVX9u1od/O8nCvWf+Bbh1hGqXJElarqyxxhrcf//9rLfeeoOtTFBLqor777+fNdZYY1jnDfkhzCEH6HwIc3ZVnfCkBpIG4YcwJUnDsbJ8CPORRx7hrrvuYt68eaNdykpvjTXW4FnPehbjxo17XPuT/RCmJEmSxpBx48axySabjHYZWkpPOoBX1TEjUIckSZK0UnAGXGNaX99E+vtXjl8nSpKklcNI7YIiSZIkqQcGcEmSJKlFBnBJkiSpRQZwSZIkqUUGcEmSJKlFBnBJkiSpRQZwSZIkqUXuA66x7Z4BODG99T28lm0tkiRJI8AZcEmSJKlFBnBJkiSpRQZwSZIkqUUGcEmSJKlFBnBJkiSpRQZwSZIkqUUGcEmSJKlF7gOusW3DPji8f7SrkCRJGjHOgEuSJEktMoBLkiRJLTKAS5IkSS0ygGtMG5g1a7RLkCRJGlEGcEmSJKlFBnBJkiSpRQZwSZIkqUUGcEmSJKlFBnBJkiSpRQZwSZIkqUUGcEmSJKlFBnCNaX0TJox2CZIkSSNqyACeZEGSmUl+neSaJC9po7DF1DItyY+ax/smOaV5fFCSdzePz0jypySrN8/XT3JH83hSkrld9/PLJFuM0u1IkiRpJdTLDPjcqppSVdsCHwM+2+vg6Vjms+xVdVpVfbOraQHw3sV0/23X/ZwJfHxZ1ydJkiQtNNxw/FTgLwufJDkyydVJrktybNM2KcnNSU4FrgF2bp5/NcmNSaYnWbPpOyXJlc3530vytKZ9RpIdmsePzWAvTpJjkhzR1fRF4MNJVhvO/UiSJEnL2lABFWDNJDOBNYCNgJcBJNkD2AzYEQhwQZJdgDuBLYD9qurgJJOafm+vqvcl+Q7wRuA/gW8CH6yqnyf5JHA0cNgI3NedwGXAu4AfLnJs0+Z+JgDjgReOwPW0jAwM3E3zs50kaSVUdfRolyCNuOEsQdkSeBXwzSQB9mi+rqUz070lnaAN8IequrJrjN9X1czm8QAwKck6wLpV9fOm/Uxglyd1N4/3GeBInniPC5egbEon7H9lBK8pSZIkLVEvM+CPqaorkqwPbEBn1vuzVfUf3X2aGe85i5z6cNfjBcCaQ1xqPn8LzmsMp8auWm9vZrrfsoRuFwDfWJrxJUmSpKUxrDXgSbYEVgXuBy4C3ptk7ebYM5M8o9exqupB4C9Jdm6a3gUsnA2/A+hrHr9pODUu4tPAEUs4/lLgt09ifEmSJGlYhrMGHDqz3u+pqgXA9CRbAVd0VqQwG3gnnRnuXr0HOC3JeOB3wH5N+wnAd5K8C/ifYYz3OFV1Y5JrgO27mheuAQ/wf8D+Szu+JEmSNFypqtGuQVqsZGLBgaNdhiRplPghTC2vkgxU1Q6DHfMvYUqSJEktMoBLkiRJLRrWLihS2/r6JtLf768fJUnSisMZcEmSJKlFBnBJkiSpRQZwSZIkqUUGcEmSJKlFBnBJkiSpRQZwSZIkqUUGcEmSJKlF7gOuse2eATgxf3t+eI1eLZIkSSPAGXBJkiSpRQZwSZIkqUUGcEmSJKlFBnBJkiSpRQZwSZIkqUUGcEmSJKlFBnBJkiSpRe4DrrFtwz44vH+0q5AkSRoxzoBLkiRJLTKAS5IkSS0ygEuSJEktMoBrTBuYNWu0S5AkSRpRBnBJkiSpRQZwSZIkqUUGcEmSJKlFBnBJkiSpRQZwSZIkqUUGcEmSJKlFBnBJkiSpRQZwjWl9EyaMdgmSJEkjasgAnqSSfKvr+WpJ/pzkRz2cO7v5PinJPl3tOyQ5aWmL7kWS1yb56BB99k1ySvP4mCR/TfKMruOzux4vSDIzya+TXJPkJcuuekmSJK2oepkBnwNsnWTN5vkrgD8N8zqTgMcCeFX1V9WhwxxjWKrqgqo6bpin3Qccvphjc6tqSlVtC3wM+OyTKlCSJEkrpV6XoPw3sHfz+O3AtxceaGaOj+h6fkOSSYucfxywczOD/OEk0xbOoDfnn55kRpLfJTm0a6x/bMa7IclhTdukJLck+VrTflaS3ZNcnuS2JDs2/bpnt1+T5FdJrk1ycZINF3OfpwNvTfL0IV6PpwJ/GaKPJEmS9ASr9djvv4CjmtC8DZ2guvMwrvNR4IiqejVAkmmLHN8S2A2YAPwmyZeb6+wHvBAI8KskP6cTfJ8HvBk4ALiazuz6S4HXAh8HXr/I+JcBL6qqSrI/8E8MPtM9u7m3DwFHL3JszSQzgTWAjYCX9XrzWnoDA3eTHDvaZUiSRNWi0UBaOj0F8Kq6rpnVfjtw4TKo48dV9TDwcJJ7gQ3pBOrvVdUcgCTn0wn9FwC/r6rrm/YbgZ814fp6OstdFvUs4JwkGwFPAX6/hFpOAmYmOXGR9rlVNaW55ouBbybZuqpqqe5YkiRJK6Xh7IJyAXACXctPGvMXGWeNpajj4a7HC+j8YJAe+z/a9fxRBv+h4mTglKqaDBy4pBqr6gHgbODgJfS5Algf2GAJNUqSJElPMJwAfjrwyYUzz13uALYHSLI9sMkg586is7xkOC4FXp9kfJK1gDcAvxjmGAutw98+OPqeHvp/nk5QH/Q3BEm2BFYF7l/KeiRJkrSS6jmAV9VdVfWlQQ6dBzy9WR/9fuDWQfpcB8xvtvD7cI/XuwY4A7gK+BXwtaq6ttd6F3EMcG6SX9DZ6WSoa98HfA9Yvat5zeZDpDOBc4D3VNWCpaxHkiRJK6m4hFljWTKxOr+MkCRpdPkhTA1HkoGq2mGwY/4lTEmSJKlFBnBJkiSpRb3uAy6Nir6+ifT3+ys/SZK04nAGXJIkSWqRAVySJElqkQFckiRJapEBXJIkSWqRAVySJElqkQFckiRJapEBXJIkSWqR+4BrbLtnAE7M0P0Or2VfiyRJ0ghwBlySJElqkQFckiRJapEBXJIkSWqRAVySJElqkQFckiRJapEBXJIkSWqRAVySJElqkfuAa2zbsA8O7x/tKiRJkkaMM+CSJElSiwzgkiRJUosM4JIkSVKLDOAa0wZmzSIzZox2GZIkSSPGAC5JkiS1yAAuSZIktcgALkmSJLXIAC5JkiS1yAAuSZIktcgALkmSJLXIAC5JkiS1yACuMa1vwgRq2rTRLkOSJGnEDBnAk1SSE7ueH5HkmGVa1eB1rJvk4EXaNk9yYZLbk9yc5DtJNlzK8Q9LMn4pzvvlYtrPSPKmpalFkiRJK65eZsAfBv4+yfojeeEkqw3zlHWBxwJ4kjWAHwNfrqrnVdVWwJeBDZaypMOAQQN4klUXd1JVvWQprydJkqSVUC8BfD7wFeDDix5IskGS85Jc3Xzt1LTvmOSXSa5tvm/RtO+b5NwkPwSmJ1kryenNudcmeV3T7wVJrkoyM8l1STYDjgM2bdqOB/YBrqiqHy6sp6ouqaobkqya5Phm3OuSHNiMOy3JjCTfTXJLkrPScSgwEbgkySVN39lJPpnkV8CLk/xjkhuar8O6XoPZzfckOSXJTUl+DDxjmO+FJEmSVgK9zkL/O3Bdkn9bpP1LwBeq6rIkzwEuArYCbgF2qar5SXYHPgO8sTnnxcA2VfX/knwG+J+qem+SdYGrklwMHAR8qarOSvIUYFXgo8DWVTUFIMnngYHF1PsPwINVNTXJ6sDlSaY3x7YDXgDcDVwO7FRVJyX5R2C3qrqv6bcWcENVHZWkD9gPeCEQ4FdJfl5V13Zd8w3AFsBkYEPgJuD0oV9aLcnAwN0kx452GZKk5VzV0aNdgvSYngJ4VT2U5JvAocDcrkO7A89PsvD5U5NMANYBzmxmrgsY13XOT6vq/zWP9wBem+SI5vkawHOAK4B/TvIs4Pyquq3rGr3YA9imaw32OsBmwP8BV1XVXQBJZgKTgMsGGWMBcF7z+KXA96pqTnPe+cDOQHcA3wX4dlUtAO5O8j/DKViSJEkrh+Gsw/4icA3wja62VYAXV1V3KCfJycAlVfWGJJOAGV2H53R3Bd5YVb9Z5Fo3N0s/9gYuSrI/8LtF+twI7LqYWgN8sKouWqSuaXTWtC+0gMW/BvOaML1wvF5Uj/0kSZK0kup5G8Jm1vo7dJZ3LDQd+MDCJ0mmNA/XAf7UPN53CcNeBHwwzfR2ku2a788FfldVJwEXANsAs4AJXeeeDbwkyd5d139VksnNuO9PMq5p3zzJWkPc4qLjd7sUeH2S8c04bwB+MUiftzXrzzcCdhviepIkSVoJDXcf8BOB7t1QDgV2aD7oeBOdtdsA/wZ8NsnldNZvL86n6CxPuS7JDc1zgLcCNzRLRLYEvllV99NZy31DkuObWfdX0wnwtzXX3xe4F/ganTXY1zTj/gdDz/Z/BfjvhR/C7FZV1wBnAFcBvwK+tsj6b4DvAbcB19PZjeXnQ1xPkiRJK6FUuWpCY1cyseDA0S5DkrSc80OYaluSgaraYbBj/iVMSZIkqUUGcEmSJKlFw/1rlFKr+vom0t/vrw0lSdKKwxlwSZIkqUUGcEmSJKlFBnBJkiSpRQZwSZIkqUUGcEmSJKlFBnBJkiSpRQZwSZIkqUXuA66x7Z4BODF/e354jV4tkiRJI8AZcEmSJKlFBnBJkiSpRQZwSZIkqUUGcEmSJKlFBnBJkiSpRQZwSZIkqUUGcEmSJKlF7gOusW3DPji8f7SrkCRJGjHOgEuSJEktMoBLkiRJLTKAS5IkSS1yDbjGtIFZs8iMGaNdxhPUtGmjXYIkSVpOOQMuSZIktcgALkmSJLXIAC5JkiS1yAAuSZIktcgALkmSJLXIAC5JkiS1yAAuSZIktainfcCT/DOwD7AAeBQ4EBgAPgm8GZjTdD23qj7dnLMAuB4YB8wHzgS+WFWPNsd3BE4ANgQKuAw4FHgLsENVfWAE7o8kFwL7VNUDSQ4F3g9cA5wDPL+qjhuJ62jZ6JswgX733JYkSSuQIQN4khcDrwa2r6qHk6wPPAX4V+DvgMlVNS/JBODwrlPnVtWUZoxnAGcD6wBHJ9kQOBd4W1VdkSTAG4EJI3drHVW1V9fTg4E9q+r3zfMLeh0nyWpVNX9Ei5MkSdJKp5clKBsB91XVwwBVdR/wAPA+4INVNa9pn1VVxww2QFXdCxwAfKAJ24cAZ1bVFc3xqqrvVtU93ecleU2SXyW5NsnFTXAnya5JZjZf1yaZkGSjJJc2bTck2bnpe0eS9ZOcBjwXuCDJh5Psm+SUps8GSc5LcnXztVPTfkySrySZDnyz95dVkiRJGlwvAXw68OwktyY5NcmuwPOAO6tqVq8XqqrfNdd7BrA1nSUsQ7kMeFFVbQf8F/BPTfsRwCHNDPvOwFw6S2Quatq2BWYucv2DgLuB3arqC4tc50vAF6pqKp2Z+K91HesDXldV+/R0o5IkSdISDLkEpapmJ+mjE3R3o7N2+jPdfZLsB3wIWA94SVX9cTHDZZj1PQs4J8lGdJa9LFw6cjnw+SRnAedX1V1JrgZOTzIO+H5VzRzGdXYHnt+ZnAfgqc2SGoALqmruMOvWCBkYuJvk2NEuQ5K0gqo6erRL0Eqop11QqmpBVc2ozn+lHwBeAzxnYUitqm80M88PAqsONkaS59L5EOe9wI10ZpaHcjJwSlVNpvPBzzWa6x0H7A+sCVyZZMuquhTYBfgT8K0k7+7l3hqrAC+uqinN1zO7ZvfnLOlESZIkaTiGDOBJtkiyWVfTFOA3wNeBU5Ks0fRblc4s9WBjbACcRidMF3AK8J4kL+zq884kf7fIqevQCdQA7+nqu2lVXV9VnwP6gS2TbAzcW1VfbWrbfqh76zKdzg8WC8efMoxzJUmSpJ71sg3h2sDJSdals53g7XQ+UPkg8CnghiSz6KzDPpPOOmuANZPM5G/bEH4L+DxAVd2T5G3ACc0OKY8ClwLnL3LtY4Bzk/wJuBLYpGk/LMludGbUbwL+G3gbcGSSR4DZwHBmwA8F/j3JdXRek0uBg4ZxviRJktSTdCakpbEpmVid1UeSJI0814BrWUkyUFU7DHbMv4QpSZIktcgALkmSJLXIAC5JkiS1qJcPYUqjpq9vIv39rs+TJEkrDmfAJUmSpBYZwCVJkqQWGcAlSZKkFhnAJUmSpBYZwCVJkqQWGcAlSZKkFhnAJUmSpBa5D7jGtnsG4MT87fnhNXq1SJIkjQBnwCVJkqQWGcAlSZKkFhnAJUmSpBYZwCVJkqQWGcAlSZKkFhnAJUmSpBYZwCVJkqQWuQ+4xrYN++Dw/tGuQpIkacQ4Ay5JkiS1yAAuSZIktcgALkmSJLXINeAa0wZmzSIzZiyz8WvatGU2tiRJ0mCcAZckSZJaZACXJEmSWmQAlyRJklpkAJckSZJaZACXJEmSWmQAlyRJklpkAJckSZJa1NM+4En+GdgHWAA8ChwIDACfBN4MzGm6nltVn27OWQBcD4wD5gNnAl+sqkeb4zsCJwAbAgVcBhwKvAXYoao+MAL3R5ILgX2q6oEkhwLvB64BzgGeX1XHjcR1tGz0TZhAv3t1S5KkFciQATzJi4FXA9tX1cNJ1geeAvwr8HfA5Kqal2QCcHjXqXOrakozxjOAs4F1gKOTbAicC7ytqq5IEuCNwISRu7WOqtqr6+nBwJ5V9fvm+QW9jpNktaqaP6LFSZIkaaXTyxKUjYD7quphgKq6D3gAeB/wwaqa17TPqqpjBhugqu4FDgA+0ITtQ4Azq+qK5nhV1Xer6p7u85K8Jsmvklyb5OImuJNk1yQzm69rk0xIslGSS5u2G5Ls3PS9I8n6SU4DngtckOTDSfZNckrTZ4Mk5yW5uvnaqWk/JslXkkwHvtn7yypJkiQNrpcAPh14dpJbk5yaZFfgecCdVTWr1wtV1e+a6z0D2JrOEpahXAa8qKq2A/4L+Kem/QjgkGaGfWdgLp0lMhc1bdsCMxe5/kHA3cBuVfWFRa7zJeALVTWVzkz817qO9QGvq6p9erpRSZIkaQmGXIJSVbOT9NEJurvRWTv9me4+SfYDPgSsB7ykqv64mOEyzPqeBZyTZCM6y14WLh25HPh8krOA86vqriRXA6cnGQd8v6pmDuM6uwPP70zOA/DUZkkNwAVVNXeYdWuEDAzcTXLsaJchSVpOVB092iVIQ+ppF5SqWlBVM6rzX/UHgNcAz1kYUqvqG83M84PAqoONkeS5dD7EeS9wI52Z5aGcDJxSVZPpfPBzjeZ6xwH7A2sCVybZsqouBXYB/gR8K8m7e7m3xirAi6tqSvP1zK7Z/TlLOlGSJEkajiEDeJItkmzW1TQF+A3wdeCUJGs0/ValM0s92BgbAKfRCdMFnAK8J8kLu/q8M8nfLXLqOnQCNcB7uvpuWlXXV9XngH5gyyQbA/dW1Veb2rYf6t66TKfzg8XC8acM41xJkiSpZ71sQ7g2cHKSdelsJ3g7nQ9UPgh8CrghySw667DPpLPOGmDNJDP52zaE3wI+D1BV9yR5G3BCs0PKo8ClwPmLXPsY4NwkfwKuBDZp2g9LshudGfWbgP8G3gYcmeQRYDYwnBnwQ4F/T3IdndfkUuCgYZwvSZIk9SSdCWlpbEomVmf1kSRJQ3MNuMaKJANVtcNgx/xLmJIkSVKLDOCSJElSiwzgkiRJUot6+RCmNGr6+ibS3+96PkmStOJwBlySJElqkQFckiRJapEBXJIkSWqRAVySJElqkQFckiRJapEBXJIkSWqRAVySJElqkfuAa2y7ZwBOzN+eH16jV4skSdIIcAZckiRJapEBXJIkSWqRAVySJElqkQFckiRJapEBXJIkSWqRAVySJElqkQFckiRJapH7gGts27APDu8f7SokSZJGjDPgkiRJUosM4JIkSVKLDOCSJElSi1wDrjFtYNYsMmPGqF2/pk0btWtLkqQVkzPgkiRJUosM4JIkSVKLDOCSJElSiwzgkiRJUosM4JIkSVKLDOCSJElSiwzgkiRJUouG3Ac8yeyqWnuRtoOAv1bVN5dZZZ3rvBf4MFB0flj4Z+BpwCur6u1d/dYHbgaeBTwKfAp4I/Aw8Ffg6Kr672VZq5aNvgkT6HcvbkmStAJZqj/EU1WnjXQh3ZIEeDadwL19VT2YZG1gA+B+4IQk46vqr80pbwIuqKqHkxwHbARs3TzfENh1WdYrSZIk9WqplqAkOSbJEc3jGUk+l+SqJLcm2blpXzXJ8UmuTnJdkgOb9rWT/CzJNUmuT/K6pn1SkpuTnApcA2wCzAJmA1TV7Kr6fVU9BFwKvKarpLcB304yHngf8MGqerg5756q+s7S3KckSZI00kZqDfhqVbUjcBhwdNP2D8CDVTUVmAq8L8kmwDzgDVW1PbAbcGIz4w2wBfDNqtoOuAy4B/h9km8k6Q7c36YTukkyEdgcuAR4HnBnE9IlSZKkMWeplqAM4vzm+wAwqXm8B7BNkjc1z9cBNgPuAj6TZBc667WfCWzY9PlDVV0JUFULkryKTnh/OfCFJH1VdQzwI+DUJE8F3gJ8t+k/QrejsWJg4G6SY0e7DEnScqTq6KE7SaNopAL4w833BV1jhs5SkIu6OybZl85a7r6qeiTJHcAazeE53X2rqoCrgKuS/BT4BnBMVc1N8hPgDXRmwj/cnHI78JwkE6pq1gjdmyRJkjRiluU2hBcB708yDiDJ5knWojMTfm8TvncDNh7s5CQTk2zf1TQF+EPX828D/0hn9nzhrPlfga8DJyV5SjPORkneOaJ3JkmSJC2lXmbAxye5q+v553sc+2t0lqNc06zx/jPweuAs4IdJ+oGZwC2LOX8cnd1OJtJZN/5n4KCu49OBM4GvNzPlC/0L8K/ATUnm0ZlVP6rHmiVJkqRlKo/PrtLYkkwsOHC0y5AkLUdcA66xIMlAVe0w2DH/EqYkSZLUIgO4JEmS1CIDuCRJktSikdqGUFom+vom0t/vWj5JkrTicAZckiRJapEBXJIkSWqRAVySJElqkQFckiRJapEBXJIkSWqRAVySJElqkQFckiRJapH7gGtsu2cATszfnh9eo1eLJEnSCHAGXJIkSWqRAVySJElqkQFckiRJapEBXJIkSWqRAVySJElqkQFckiRJapEBXJIkSWqR+4BrbNuwDw7vH+0qJEmSRowz4JIkSVKLDOCSJElSiwzgkiRJUotcA64xbWDWLDJjxqDHatq0VmuRJEkaCc6AS5IkSS0ygEuSJEktMoBLkiRJLTKAS5IkSS0ygEuSJEktMoBLkiRJLTKAS5IkSS3qaR/wJP8M7AMsAB4FDgQGgE8CbwbmNF3PrapPN+csAK4HxgHzgTOBL1bVo83xHYETgA2BAi4DDgXeAuxQVR8YgfsjyYXAPlX1QJJDgfcD1wDnAM+vquNG4jpaNvomTKDf/b4lSdIKZMgAnuTFwKuB7avq4STrA08B/hX4O2ByVc1LMgE4vOvUuVU1pRnjGcDZwDrA0Uk2BM4F3lZVVyQJ8EZgwsjdWkdV7dX19GBgz6r6ffP8gl7HSbJaVc0f0eIkSZK00ullCcpGwH1V9TBAVd0HPAC8D/hgVc1r2mdV1TGDDVBV9wIHAB9owvYhwJlVdUVzvKrqu1V1T/d5SV6T5FdJrk1ycRPcSbJrkpnN17VJJiTZKMmlTdsNSXZu+t6RZP0kpwHPBS5I8uEk+yY5pemzQZLzklzdfO3UtB+T5CtJpgPf7P1llSRJkgbXyxKU6cBRSW4FLqazdOMvwJ1VNavXC1XV75KsAjwD2JrOkpShXAa8qKoqyf7AP9GZZT8COKSqLk+yNjCPTsC/qKo+nWRVYPwi1z8oyauA3arqviT7dh3+EvCFqrosyXOAi4CtmmN9wEuram6v96qRMzBwN8mxo12GJGklUXX0aJeglcCQAbyqZifpA3YGdqMTwD/T3SfJfsCHgPWAl1TVHxczXIZZ37OAc5JsRGfZy8KlI5cDn09yFnB+Vd2V5Grg9CTjgO9X1cxhXGd34PmdyXkAntosqQG4wPAtSZKkkdLTLihVtaCqZlTnx8IPAK8BnrMwpFbVN5r13g8Cqw42RpLn0vkQ573AjXRmlodyMnBKVU2m88HPNZrrHQfsD6wJXJlky6q6FNgF+BPwrSTv7uXeGqsAL66qKc3XM7tm9+cs6URJkiRpOIYM4Em2SLJZV9MU4DfA14FTkqzR9FuVziz1YGNsAJxGJ0wXcArwniQv7OrzziR/t8ip69AJ1ADv6eq7aVVdX1WfA/qBLZNsDNxbVV9tatt+qHvrMp3ODxYLx58yjHMlSZKknvWyBnxt4OQk69LZTvB2OuutHwQ+BdyQZBYwl8667rub89ZMMpO/bUP4LeDzAFV1T5K3ASc0O6Q8ClwKnL/ItY8Bzk3yJ+BKYJOm/bAku9GZUb8J+G/gbcCRSR4BZgPDmQE/FPj3JNfReU0uBQ4axvmSJElST9KZkJbGpmRidVYfSZK07PkhTI2UJANVtcNgx/xLmJIkSVKLDOCSJElSi3r6U/TSaOnrm0h/v78OlCRJKw5nwCVJkqQWGcAlSZKkFhnAJUmSpBYZwCVJkqQWGcAlSZKkFhnAJUmSpBYZwCVJkqQWGcA1tt0zACem8yVJkrQCMIBLkiRJLTKAS5IkSS0ygEuSJEktMoBLkiRJLTKAS5IkSS0ygEuSJEktMoBLkiRJLVpttAuQlmjDPji8f7SrkCRJGjHOgEuSJEktMoBLkiRJLTKAS5IkSS1yDbjGtIFZs8iMGUt9fk2bNmK1SJIkjQRnwCVJkqQWGcAlSZKkFhnAJUmSpBYZwCVJkqQWGcAlSZKkFhnAJUmSpBYZwCVJkqQWuQ+4xrS+CRPody9vSZK0AhlyBjzJ7EHaDkry7mVT0uOu894k1ye5LskNSV6XZN8k316k3/pJ/pxk9STjkhyX5LbmnKuS7Lmsa5UkSZJ6sVQz4FV12kgX0i1JgGcD/wxsX1UPJlkb2AC4Hzghyfiq+mtzypuAC6rq4STHARsBWzfPNwR2XZb1SpIkSb1aqjXgSY5JckTzeEaSzzUzzbcm2blpXzXJ8UmubmawD2za107ysyTXNLPbr2vaJyW5OcmpwDXAJsAsYDZAVc2uqt9X1UPApcBrukp6G/DtJOOB9wEfrKqHm/PuqarvLM19SpIkSSNtpNaAr1ZVOybZCzga2B34B+DBqpqaZHXg8iTTgT8Cb6iqh5KsD1yZ5IJmnC2A/arq4CSrAvcAv0/yM+D8qvph0+/bwD7AOUkmApsDlwAvAO5sQrpWAAMDd5McO9plSJJWQlVHj3YJWkGN1C4o5zffB4BJzeM9gHcnmQn8ClgP2AwI8Jkk1wEXA88ENmzO+UNVXQlQVQuAV9FZXnIr8IUkxzT9fgS8NMlTgbcA3236S5IkSWPaSM2AP9x8X9A1ZugsBbmou2OSfems5e6rqkeS3AGs0Rye0923qgq4CrgqyU+BbwDHVNXcJD8B3kBn+cmHm1NuB56TZEJVzRqhe5MkSZJGzLLcB/wi4P1JxgEk2TzJWsA6wL1N+N4N2Hiwk5NMTLJ9V9MU4A9dz78N/COd2fOFs+Z/Bb4OnJTkKc04GyV554jemSRJkrSUepkBH5/krq7nn+9x7K/RWY5yTbOryZ+B1wNnAT9M0g/MBG5ZzPnj6Ox2MhGY15x/UNfx6cCZwNebmfKF/gX4V+CmJPPozKof1WPNkiRJ0jKVx2dXaWxJJhYcONplSJJWQn4IU09GkoGq2mGwY/4pekmSJKlFBnBJkiSpRSO1C4q0TPT1TaS/318BSpKkFYcz4JIkSVKLDOCSJElSiwzgkiRJUosM4JIkSVKLDOCSJElSiwzgkiRJUosM4JIkSVKL3AdcY9s9A3BiRrsKSWPR4TXaFUjSUnEGXJIkSWqRAVySJElqkQFckiRJapEBXJIkSWqRAVySJElqkQFckiRJapEBXJIkSWqR+4BrbNuwDw7vH+0qJEmSRowz4JIkSVKLDOCSJElSiwzgkiRJUotcA64xbWDWLDJjxmiXIUmSVhA1bdpol+AMuCRJktQmA7gkSZLUIgO4JEmS1CIDuCRJktQiA7gkSZLUIgO4JEmS1CIDuCRJktQi9wHXmNY3YQL9Y2C/TkmSpJEy5Ax4kgVJZia5IckPk6w7EhdOsm+SU0ZorDuSXN/UOTPJS0Zi3EGuMyXJXou07ZmkP8nNSW5JckLTfkySI0bw2r/senx8khub7wclefdIXUeSJEnLVi8z4HOragpAkjOBQ4BPL8uiltJuVXXfcE5IslpVzR/GKVOAHYALm/O3Bk4B9q6qW5KsBhwwnBp6VVXdP1QcCGxQVQ8Pd5yluGdJkiSNoOGuAb8CeCZAkh2T/DLJtc33LZr2fZOcn+QnSW5L8m8LT06yX5Jbk/wc2KmrfeMkP0tyXfP9OU37GUm+nOSSJL9LsmuS05vZ5jOWVOgQY34+ySXA55Js2tQ6kOQXSbZs+r25mfX/dZJLkzwF+CTw1maW/a3APwGfrqpbAKpqflWdOkgt70tydTPWeUnGD3aNpu0FSa5qrnFdks2a9tnN9wuAtYBfJXlr90z7Eu7lcfc8jPdbkiRJI6znNeBJVgVeDny9aboF2KWq5ifZHfgM8Mbm2BRgO+Bh4DdJTgbmA8cCfcCDwCXAtU3/U4BvVtWZSd4LnAS8vjn2NOBlwGuBH9IJ7vsDVyeZUlUzm36XJFkAPFxVLxxizM2B3atqQZKfAQdV1W1JXgic2lzvKOCVVfWnJOtW1f8lOQrYoao+0LwmHwFO7OHlO7+qvtqc86/APwAnL3qNpu9BwJeq6qwm9K/aPVBVvTbJ7K7fShzTdfgri7mXx91zD/WOGQMDd5McO9plSJJaUnX0aJcgLXO9BPA1k8wEJgEDwE+b9nWAM5sZ2gLGdZ3zs6p6ECDJTcDGwPrAjKr6c9N+Dp1QCPBi4O+bx98C/q1rrB9WVSW5Hrinqq5vzr+xqWlm02/RJShLGvPcJnyvDbwEODfJwmOrN98vB85I8h3g/CW8Pr3Yugne6wJrAxct4RpXAP+c5Fl0gvttvVxgiHuB5p6f1F1IkiTpSetlCcrCNeAbA0+hswYc4FPAJVW1NfAaYI2uc7rXJi/gb0G/eqyru9/CsR5dZNxHGd4uLt1jzmm+rwI8UFVTur62Aqiqg4B/AZ4NzEyy3iBj3khnRn8oZwAfqKrJdH4LsMbirlFVZ9OZ7Z8LXJTkZYMP+QSLvZdF7lmSJEmjqOc14M2M9qHAEUnG0ZkB/1NzeN8ehvgVMC3Jes35b+469kvgbc3jdwCX9VrXEgw5ZlU9BPw+yZsB0rFt83jTqvpVVR0F3EcnJM8CJnQNcTzw8SSbN+eskuQfB6llAvC/zX2/Y2HjYNdI8lzgd1V1EnABsE0vN7uke5EkSdLYMawPYVbVtcCv6QTbfwM+m+RyFlmnvJhz/xc4hs4Si4uBa7oOHwrsl+Q64F3Ah4ZT12L0OuY7gH9I8ms6M9qva9qPT2drwxuAS+nc9yXA8xd+CLOqrgMOA76d5GbgBmCjQa7xCTo/gPyUztr5hQa7xluBG5plP1sC3xzGPS/uXiRJkjRGpKrXVSFS+5KJ1dl1UZK0MvBDmFpRJBmoqh0GO+afopckSZJaZACXJEmSWjScXUSk1vX1TaS/319HSpKkFYcz4JIkSVKLDOCSJElSiwzgkiRJUosM4JIkSVKLDOCSJElSiwzgkiRJUosM4JIkSVKL3AdcY9s9A3BiRrsKSZK0oji8RrsCZ8AlSZKkNhnAJUmSpBYZwCVJkqQWGcAlSZKkFhnAJUmSpBYZwCVJkqQWGcAlSZKkFrkPuMa2Dfvg8P7RrkKSJGnEOAMuSZIktcgALkmSJLXIAC5JkiS1yAAuSZIktcgALkmSJLXIAC5JkiS1yAAuSZIktcgALkmSJLXIAC5JkiS1yAAuSZIktcgALkmSJLXIAC5JkiS1aMgAnmR21+O9ktyW5DlJjkny1yTPGKzvEsa7MMm6Q/SZkWSHQdr3TXLKUNdYGkmOSHJLkhuS/DrJu5dUy1JeY4ckJzWPV09ycZKZSd6a5GtJnj8S15EkSdLYtVqvHZO8HDgZ2KOq7kwCcB9wOPCRXsepqr2GW+RISKfgVNWjgxw7CHgFsGNVPZRkHeD1I11DVfUD/c3T7YBxVTWleX7OcMZKsmpVLRjB8iRJktSCnpagJNkZ+Cqwd1X9tuvQ6cBbkzx9kHPemeSqZob3P5Ks2rTfkWT95vEnmlnnnyb5dpIjuoZ4c3P+rc31F3p2kp8k+U2So7uu94/N7PUNSQ5r2iYluTnJqcA1zblnNH2uT/Lh5vSPAwdX1UMAVfVgVZ05yD19OUl/khuTHNvVflySm5Jcl+SEpu3NXbPplzZt05L8qPmtwX8CU5rXZ9PumfYkeyS5Isk1Sc5NsnbXa3dUksuANw/1vkmSJGns6WUGfHXgB8C0qrplkWOz6YTwDwHdYXgr4K3ATlX1SBOA3wF8s6vPDsAb6cwEr0YnIA9011ZVOybZqxl796Z9R2Br4K/A1Ul+DBSwH/BCIMCvkvwc+AuwBbBfVR2cpA94ZlVt3dSwbpIJwIRFfrBYnH+uqv/X/DDxsyTbAHcBbwC2rKrqWl5zFPDKqvrToktuqureJPsDR1TVq5taFr4u6wP/AuxeVXOSfAT4R+CTzenzquqlPdS6QhgYuJuun3WkJ6g6euhOkiSNIb3MgD8C/BL4h8UcPwl4T5KndrW9HOijE5BnNs+fu8h5LwV+UFVzq2oW8MNFjp/ffB8AJnW1/7Sq7q+quU2flzZf36uqOVU1u2lfOGv+h6q6snn8O+C5SU5O8irgITqBvZb0AnR5S5JrgGuBFwDPb8aYB3wtyd/T+cEA4HLgjCTvA1btcXyAFzXjXt68du8BNu46PqylKpIkSRpbegngjwJvAaYm+fiiB6vqAeBs4OCu5gBnVtWU5muLqjpmkVMzxHUfbr4v4PEz9YuG5RpirDldtf4F2BaYARwCfK1ZdjInyaI/IDy+2GQT4Ajg5VW1DfBjYI2qmk9nVv48OuvGf9Jc6yA6M9nPBmYmWW9J43dfis4PGQtfu+dXVfcPP3MWd6IkSZLGvp7WgFfVX4FXA+9IMthM+OeBA/lbUP4Z8KaFO6QkeXqSjRc55zLgNUnWaNY4791jza9oxluTTuC9HLgUeH2S8UnWorMk5BeLntgs71ilqs4DPgFs3xz6LPDvC2fxkzw1yQGLnP5UOuH3wSQbAns2fdcG1qmqC4HDgClN+6ZV9auqOorOh1Wf3eP9XQnslOR5zTjjk2ze47mSJEka43reBaVZ+/wq4NIk9y1y7L4k3wM+3Dy/Kcm/ANOTrEJnGcshwB+6zrk6yQXAr5v2fuDBHkq5DPgW8Dzg7GZnEZKcAVzV9PlaVV2bZNIi5z4T+EZTE8DHmu9fBtams2TmkabeExe5x18nuRa4kc5SlsubQxOAHyRZg87s9cIPdh6fZLOm7WfNfe461M1V1Z+T7At8O8nqTfO/ALcOda4kSZLGvlT1uvx5GVw8WbuqZicZT2cW+4CqumbUCtKYk0yszi9XpMH5IUxJ0liUZKCqBv1bMj3PgC8jX0nnj8+sQWfNuOFbkiRJK7RRDeBVtc9oXl+SJElq22jPgEtL1Nc3kf5+lxhIkqQVR0+7oEiSJEkaGQZwSZIkqUUGcEmSJKlFBnBJkiSpRQZwSZIkqUUGcEmSJKlFBnBJkiSpRe4DrrHtngE4MY9vO7xGpxZJkqQR4Ay4JEmS1CIDuCRJktQiA7gkSZLUIgO4JEmS1CIDuCRJktQiA7gkSZLUIgO4JEmS1CL3AdfYtmEfHN4/2lVIkiSNGGfAJUmSpBYZwCVJkqQWGcAlSZKkFrkGXGPawKxZZMaM0S5D0gqupk0b7RIkrUScAZckSZJaZACXJEmSWmQAlyRJklpkAJckSZJaZACXJEmSWmQAlyRJklpkAJckSZJa5D7gGtP6Jkyg3/15JUnSCmTIGfAks5/sRZLskOSkJRyflGSfXvs3fe5Icn2S65L8PMnGT7bOkZLkoCTvHu06JEmSNPa0sgSlqvqr6tAldJkEPBbAe+i/0G5VtQ0wA/iXJ1UkkI4n/ZpU1WlV9c0nO44kSZJWPEsVNpNMSXJlM/v8vSRPa9qnNm1XJDk+yQ1N+7QkP2oe75pkZvN1bZIJwHHAzk3bhxfpv3aSb3TNdr9xkJKuAJ7Z9N8gyXlJrm6+dupq/2mSa5L8R5I/JFm/mX2/OcmpwDXAs5Mc2Zx7XZJjm/PXSvLjJL9OckOStzbtxyW5qel7QtN2TJIjhnitZiT5XJKrktyaZOeleS8kSZK0fFnaNeDfBD5YVT9P8kngaOAw4BvAAVX1yyTHLebcI4BDquryJGsD84CPAkdU1auhE9i7+n8CeLCqJjfHnjbImK8Cvt88/hLwhaq6LMlzgIuArZoa/6eqPpvkVcABXedvAexXVQcn2QPYDNgRCHBBkl2ADYC7q2rvpo51kjwdeAOwZVVVknWH8VoBrFZVOybZq2nffTGv2UprYOBump+BJEkatqqjR7sE6QmGPQOeZB1g3ar6edN0JrBLEz4nVNUvm/azFzPE5cDnkxzajDN/iEvuDvz7widV9ZeuY5ckubfpc3ZX/1OSzAQuAJ7azLK/FPivZoyfAN3j/KGqrmwe79F8XUtnRnxLOoH8emD3ZtZ656p6EHiIzg8QX0vy98Bfuwtf3GvV1eX85vsAnWU4kiRJWsGN5Brw9NKpqo4D9gfWBK5MsmUP49Ziju0GbAzcCHyyaVsFeHFVTWm+nllVs4aob84i1/ts1/nPq6qvV9WtQB+dIP7ZJEc1PzzsCJwHvB74yRD3sqiHm+8LcEcaSZKklcKwA3gz8/uXrjXL7wJ+3sxMz0ryoqb9bYOdn2TTqrq+qj4H9NOZYZ4FTFjMJacDH+g6/3FLUKpqLp0lHe9uloQs2n9K8/Ay4C1N2x7AYEtZoLNk5b3N8hiSPDPJM5JMBP5aVf8JnABs3/RZp6oubGqY0j3Q4l6rxVxXkiRJK4FeZl3HJ7mr6/nngfcApyUZD/wO2K859g/AV5PMobMzyYODjHdYkt3ozPreBPw38CgwP8mvgTPoLP9Y6F+Bf28+0LkAOJa/Ld0AoKr+N8m3gUOAQ5v+1zX3dylwUHPet5sPT/4c+F86wX/tRcaanmQr4IokALOBdwLPA45P8ijwCPB+Oj80/CDJGnRmzj88yP0u7rWSJEnSSihVi1vdsRSDJWtX1ezm8UeBjarqQyN2gSchyerAgqqan+TFwJerasool6UhJBMLDhztMiRJyyk/hKnRkmSgqnYY7NhIrzveO8nHmnH/AOw7wuM/Gc8BvpPOPt//B7xvlOuRJEnSSmhEA3hVnQOcM5JjjpSqug3YbrTrkCRJ0srNnTc0pvX1TaS/318fSpKkFUcrf4pekiRJUocBXJIkSWqRAVySJElqkQFckiRJapEBXJIkSWqRAVySJElqkQFckiRJapEBXJIkSWqRAVySJElqkQFckiRJapEBXJIkSWqRAVySJElqkQFckiRJapEBXJIkSWqRAVySJElqkQFckiRJapEBXJIkSWqRAVySJElqkQFckiRJapEBXJIkSWqRAVySJElqkQFckiRJapEBXJIkSWpRqmq0a5AWK8ks4DejXYd6sj5w32gXoZ74Xi0/fK+WH75Xy4+23quNq2qDwQ6s1sLFpSfjN1W1w2gXoaEl6fe9Wj74Xi0/fK+WH75Xy4+x8F65BEWSJElqkQFckiRJapEBXGPdV0a7APXM92r54Xu1/PC9Wn74Xi0/Rv298kOYkiRJUoucAZckSZJaZACXJEmSWmQA16hL8qokv0lye5KPDnI8SU5qjl+XZPvRqFM9vVfvaN6j65L8Msm2o1Gnhn6vuvpNTbIgyZvarE9/08t7lWRakplJbkzy87ZrVEcP/w9cJ8kPk/y6ea/2G406BUlOT3JvkhsWc3xUs4UBXKMqyarAvwN7As8H3p7k+Yt02xPYrPk6APhyq0UK6Pm9+j2wa1VtA3yKMfBBl5VRj+/Vwn6fAy5qt0It1Mt7lWRd4FTgtVX1AuDNbdepnv9dHQLcVFXbAtOAE5M8pdVCtdAZwKuWcHxUs4UBXKNtR+D2qvpdVf0f8F/A6xbp8zrgm9VxJbBuko3aLlRDv1dV9cuq+kvz9ErgWS3XqI5e/l0BfBA4D7i3zeL0OL28V/sA51fVnQBV5fs1Onp5rwqYkCTA2sD/A+a3W6YAqupSOq//4oxqtjCAa7Q9E/hj1/O7mrbh9tGyN9z34R+A/16mFWlxhnyvkjwTeANwWot16Yl6+Xe1OfC0JDOSDCR5d2vVqVsv79UpwFbA3cD1wIeq6tF2ytMwjWq28E/Ra7RlkLZF98bspY+WvZ7fhyS70QngL12mFWlxenmvvgh8pKoWdCbrNEp6ea9WA/qAlwNrAlckubKqbl3WxelxenmvXgnMBF4GbAr8NMkvquqhZVybhm9Us4UBXKPtLuDZXc+fRWfmYLh9tOz19D4k2Qb4GrBnVd3fUm16vF7eqx2A/2rC9/rAXknmV9X3W6lQC/X6/8D7qmoOMCfJpcC2gAG8Xb28V/sBx1Xnj6zcnuT3wJbAVe2UqGEY1WzhEhSNtquBzZJs0nxQ5W3ABYv0uQB4d/OJ5RcBD1bV/7ZdqIZ+r5I8BzgfeJezc6NqyPeqqjapqklVNQn4LnCw4XtU9PL/wB8AOydZLcl44IXAzS3Xqd7eqzvp/KaCJBsCWwC/a7VK9WpUs4Uz4BpVVTU/yQfo7MKwKnB6Vd2Y5KDm+GnAhcBewO3AX+nMMKhlPb5XRwHrAac2M6vzq2qH0ap5ZdXje6UxoJf3qqpuTvIT4DrgUeBrVTXo1mpadnr8d/Up4Iwk19NZ4vCRqrpv1IpeiSX5Np2daNZPchdwNDAOxka28E/RS5IkSS1yCYokSZLUIgO4JEmS1CIDuCRJktQiA7gkSZLUIgO4JEmS1CIDuCRJktQiA7gkSZLUov8PmoOU+kb4K5QAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 864x576 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# make some plots\n",
    "\n",
    "indices = np.arange(len(results))\n",
    "\n",
    "results = [[x[i] for x in results] for i in range(4)]\n",
    "\n",
    "clf_names, score, training_time, test_time = results\n",
    "training_time = np.array(training_time) / np.max(training_time)\n",
    "test_time = np.array(test_time) / np.max(test_time)\n",
    "\n",
    "plt.figure(figsize=(12, 8))\n",
    "plt.title(\"Score\")\n",
    "plt.barh(indices, score, .2, label=\"score\", color='navy')\n",
    "plt.barh(indices + .3, training_time, .2, label=\"training time\",\n",
    "         color='c')\n",
    "plt.barh(indices + .6, test_time, .2, label=\"test time\", color='darkorange')\n",
    "plt.yticks(())\n",
    "plt.legend(loc='best')\n",
    "plt.subplots_adjust(left=.25)\n",
    "plt.subplots_adjust(top=.95)\n",
    "plt.subplots_adjust(bottom=.05)\n",
    "\n",
    "for i, c in zip(indices, clf_names):\n",
    "    plt.text(-.3, i, c)\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3dd18d2d",
   "metadata": {},
   "source": [
    "本节完。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5b57458f",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}